/*******************************************************************************
Project:        Wealth -- Smith, Zidar, and Zwick
Last modified:  2021-07-06
Modified by:    Dustin Swonder
Description:    This file loads and processes Distributional Financial Accounts
				data for wise use in revision exhibits. 
*******************************************************************************/

/*******************************************************************************
	(0) Load SCF data to help portfolio category divisions
*******************************************************************************/
		
	/***************************************************************************
		(0.1) Load data and collapse to yield aggregates
	***************************************************************************/

use $dtadir/scf_revision.dta, clear

#delimit ;
keep year wgt currency pthru ccorw_mutf mmda saving call cds stocks notxbnd tfbmutf 
	privccorw stmutf comutf gbmutf obmutf mmmf omutf cashli retqliq tot_pen_db 
	houses oresre mrthel resdbt othloc ccbal odebt durables privloans 
	mortgageassets vehic govtbnd irakh scorw obnd trusts_equity trusts_mmbondfund 
	install pthrubus_costbasis scorw_costbasis privccorw_costbasis nnresre;
#delimit cr

	/***************************************************************************
		(0.2) Create DFA analog concepts in SCF
	***************************************************************************/

gen realestate_scf = houses + oresre
gen consumerdurables_scf = vehic + durables
gen timedepositsandshortterminve_scf = saving + cds
gen moneymarketfundshares_scf = mmda + mmmf
gen govtmunicipalsecurities_scf = notxbnd + govtbnd
gen otherloansandadvances_scf = call + privloans

/* For closely held biz, average of market values and cost basis; note that 
	privccorw (market value) and mmmf are in ccorw_mutf */
gen corpequmutf_scf = ccorw_mutf + 0.5 * (privccorw_costbasis - privccorw) + ///
					  0.5 * (scorw + scorw_costbasis) - mmmf + tfbmutf

gen pensionentitlements_scf = retqliq - irakh + tot_pen_db

* Business values are midpoints of market value and cost basis
gen equityinnoncorpoatebusiness_scf = 0.5 * (pthrubus_costbasis + nnresre - scorw_costbasis) + ///
									  0.5 * (pthru - scorw)

gen homemortgages_scf = mrthel + resdbt
gen consumercredit_scf = install + ccbal

rename (currency obnd mortgageassets othloc odebt) ///
	(checkabledepostsandcurrency_scf corporateandforeignbonds_scf mortgages_scf ///
		depinstitutionsloansnec_scf otherloansandadvancesliab_scf)

foreach liability of varlist homemortgages_scf depinstitutionsloansnec_scf ///
	consumercredit_scf otherloansandadvancesliab_scf {
	assert sign(`liability') == 1 | `liability' == 0
}

#delimit ;
gen networth_pref = realestate_scf + consumerdurables_scf + 
					timedepositsandshortterminve_scf + moneymarketfundshares_scf + 
					govtmunicipalsecurities_scf + otherloansandadvances_scf + 
					corpequmutf_scf + cashli + pensionentitlements_scf + 
					equityinnoncorpoatebusiness_scf + checkabledepostsandcurrency_scf + 
					irakh + corporateandforeignbonds_scf + mortgages_scf - 
					homemortgages_scf - consumercredit_scf - 
					depinstitutionsloansnec_scf - otherloansandadvancesliab_scf;

gen assets_incl_iras_scf = timedepositsandshortterminve + moneymarketfundshares + 
						   govtmunicipalsecurities_scf + corporateandforeignbonds_scf + 
						   corpequmutf_scf + irakh;
#delimit cr

	/***************************************************************************
		(0.3) Rank by SCF-reconciled net worth and get aggregates by group; then
			scale into millions to match DFA
	***************************************************************************/

cumul networth_pref [aw = wgt], by(year) gen(dfanetworth_rank)

gen category = cond(dfanetworth_rank > 0.99, "Top1", ///
			   cond(dfanetworth_rank > 0.9, "Next9", ///
			   cond(dfanetworth_rank > 0.5, "Next40", "Bottom50")))

qui ds year category wgt dfanetworth_rank, not 
local collapselist = "`r(varlist)'"

foreach unweighted of varlist `collapselist' {
	replace `unweighted' = `unweighted' * wgt
}

collapse (sum) `collapselist', by(year category)

foreach unscaled of varlist `collapselist' { // Scale into M to match DFA/parameters
	qui replace `unscaled' = `unscaled' / 1E6 
}

	/***************************************************************************
		(0.4) Decompose shares of DFA analogs
	***************************************************************************/

gen ira_sh_assets_incl_iras_scf = irakh / assets_incl_iras_scf

gen pubccorp_sh_corpequmutf = (stocks + trusts_equity) / corpequmutf_scf
gen privccorp_sh_corpequmutf = 0.5 * (privccorw + privccorw_costbasis) / corpequmutf_scf
gen scorp_sh_corpequmutf = 0.5 * (scorw + scorw_costbasis) / corpequmutf_scf
gen equmutf_sh_corpequmutf = (stmutf + (0.5 * comutf) + (0.5 * omutf)) / corpequmutf_scf
gen inttaxmutf_sh_corpequmutf = ((0.5 * comutf) + gbmutf + obmutf + (0.5 * omutf) + trusts_mmbondfund) / corpequmutf_scf
gen intexmmutf_sh_corpequmutf = tfbmutf / corpequmutf_scf

egen checktotal = rowtotal(*_sh_corpequmutf)
assert inrange(checktotal, 0.99, 1.01)
drop checktotal

gen intexm_sh_govtmunis = notxbnd / govtmunicipalsecurities_scf

gen mmda_share_moneymarket = mmda / moneymarketfundshares_scf

tempfile scf
save `scf'

/********************************************************************************
	(1) Load in raw DFA data and generate year variable
********************************************************************************/

insheet using "$rawdir/dfa20210216/dfa-networth-levels-detail.csv", clear

isid date category

gen year = real(substr(date, 1, 4))
assert !missing(year)
keep if inrange(year, 1989, 2019) // drop 2020 rows

/********************************************************************************
	(2) Collapse over years and categories to yield annual aggregates
********************************************************************************/

ds year date category, not
local collapselist = "`r(varlist)'"
collapse (mean) `collapselist', by(category year)

/********************************************************************************
	(3) Adjust DFA portfolio categories that need to be adjusted
********************************************************************************/

	/****************************************************************************
		(3.1) Remove IRAs
	****************************************************************************/

#delimit ;
gen assets_incl_iras = timedepositsandshortterminve + moneymarketfundshares + 
					   usgovernmentandmunicipalsecuriti + corporateandforeignbonds + 
					   corporateequitiesandmutualfundsh;
#delimit cr

merge 1:1 category year using `scf', assert(1 3) keepusing(ira_sh_assets_incl_iras)
assert mod(year - 1989, 3) > 0 if _merge == 1
assert _merge == 3 if mod(year - 1989, 3) == 0
drop _merge

ipolate ira_sh_assets_incl_iras year, gen(ira_shares_iplt)

gen peniraw_dfa = ira_shares_iplt * assets_incl_iras

gen todistribute_dfa = assets_incl_iras - peniraw_dfa

foreach dfaconcept of varlist timedepositsandshortterminvestme moneymarketfundshares ///
	usgovernmentandmunicipalsecuriti corporateandforeignbonds ///
	corporateequitiesandmutualfundsh {

	#delimit ;
	local newname = cond("`dfaconcept'" == "timedepositsandshortterminvestme", "timdepshrttrm_excl_iras", 
				  cond("`dfaconcept'" == "moneymarketfundshares", "mnymrktfundshares_excl_iras", 
				  cond("`dfaconcept'" == "usgovernmentandmunicipalsecuriti", "usgovsecmunishares_excl_iras", 
				  cond("`dfaconcept'" == "corporateandforeignbonds", "corpfrgnbnd_excl_iras", 
				  cond("`dfaconcept'" == "corporateequitiesandmutualfundsh", "corpequmutf_excl_iras", "alpaca")))));
	#delimit cr
	assert "`newname'" != "alpaca"

	gen share = `dfaconcept' / assets_incl_iras

	gen `newname' = share * todistribute_dfa

	drop share
}

gen ira_timdepshrttrm = timedepositsandshortterminvestme - timdepshrttrm_excl_iras
gen ira_mnymrktfundshares = moneymarketfundshares - mnymrktfundshares_excl_iras
gen ira_usgovsecmunishares = usgovernmentandmunicipalsecuriti - usgovsecmunishares_excl_iras
gen ira_corpfrgnbnd = corporateandforeignbonds - corpfrgnbnd_excl_iras
gen ira_corpequmutf = corporateequitiesandmutualfundsh - corpequmutf_excl_iras

drop todistribute_dfa ira_sh_assets_incl_iras ira_shares_iplt assets_incl_iras

	/****************************************************************************
		(3.2) Disaggregate IRA-adjusted corporate equities into different 
			pieces: 
				a. Public C-corporation equity
				b. Private C-corporation equity
				c. S-corporation equity
				d. Mutual funds invested in fixed claims
				e. Mutual funds invested in equity
	****************************************************************************/

merge 1:1 category year using `scf', assert(1 3) keepusing(*_sh_corpequmutf) nogen

foreach ipolatevar of varlist *_sh_corpequmutf {
	ipolate `ipolatevar' year, gen(`ipolatevar'_iplt)
}

foreach constituent in pubccorp privccorp scorp equmutf inttaxmutf intexmmutf {
	gen `constituent'_dfa = `constituent'_sh_corpequmutf_iplt * corpequmutf_excl_iras
}

drop *_sh_corpequmutf *_iplt

	/****************************************************************************
		(3.3) Separate mmda from mmmf
	****************************************************************************/

merge 1:1 category year using `scf', assert(1 3) keepusing(mmda_share_moneymarket) nogen

ipolate mmda_share_moneymarket year, gen(mmda_share_moneymarket_iplt)

gen mmda_dfa = mmda_share_moneymarket_iplt * mnymrktfundshares_excl_iras
gen mmmf_dfa = mnymrktfundshares_excl_iras - mmda_dfa

drop mmda_share_moneymarket mmda_share_moneymarket_iplt

	/****************************************************************************
		(3.4) Separate munis from US government securities
	****************************************************************************/

merge 1:1 category year using `scf', assert(1 3) keepusing(intexm_sh_govtmunis) nogen

ipolate intexm_sh_govtmunis year, gen(intexm_sh_govtmunis_iplt)

gen notxbnd_dfa = intexm_sh_govtmunis_iplt * usgovsecmunishares_excl_iras
gen govtbnd_dfa = usgovsecmunishares_excl_iras - notxbnd_dfa

drop intexm_sh_govtmunis intexm_sh_govtmunis_iplt

/********************************************************************************
	(4) Allocate to flows- and risk-based portfolio categories
********************************************************************************/

gen networth_pref = networth - consumerdurables

	/****************************************************************************
		(4.1) Flows-based, imitating taxonomy for SCF
	****************************************************************************/

#delimit ;
assert sign(homemortgages) == 1 & sign(otherloansandadvancesliabilities) == 1 & 
	   sign(consumercredit) == 1 & sign(deferredandunpaindlifeinsurancep) == 1;

/* Honestly not sure why this debt concept is negative in a handful of years; 
	not explained in DFA paper */
assert sign(depositoryinstitutionsloansnec) == 1 | 
	   depositoryinstitutionsloansnec == 0 if !inlist(year, 2002, 2003, 2005);

gen inttaxw_dfa = otherloansandadvancesassets + mortgages + timdepshrttrm_excl_iras + 
			  mmda_dfa + corpfrgnbnd_excl_iras + govtbnd_dfa;
gen othdebt_dfa = - otherloansandadvancesliabilities - depositoryinstitutionsloansnec - 
				deferredandunpaindlifeinsurancep - consumercredit;
#delimit cr

gen intexmw_dfa = notxbnd_dfa + intexmmutf_dfa
gen pthru_dfa = equityinnoncorpoatebusiness + scorp_dfa

* Sometimes we want to break out C-corporation equity separately
gen ccorw_dfa = pubccorp_dfa + privccorp_dfa + equmutf_dfa
gen mmbondfund_dfa = inttaxmutf_dfa + mmmf_dfa
gen ccorw_mutf_dfa = ccorw_dfa + mmbondfund_dfa

gen hwpen_dfa = lifeinsurancereserves + pensionentitlements + peniraw_dfa
gen hwhou_dfa = realestate - homemortgages

rename (checkabledepostsandcurrency miscellaneousassets) (currency_dfa hwoth_dfa)

#delimit ;
gen networth_check_flows = currency_dfa + inttaxw_dfa + intexmw_dfa + othdebt_dfa +
						   pthru_dfa + ccorw_mutf_dfa + hwpen_dfa + hwhou_dfa +
						   hwoth_dfa;

assert inrange(networth_check_flows / networth_pref, 0.9999, 1.0001) | 
	   abs(networth_check_flows - networth_pref) < 1;
#delimit cr

	/****************************************************************************
		(4.2) Risk-based, imitating taxonomy for SCF
	****************************************************************************/

gen hwfix_dfa = currency_dfa + inttaxw_dfa + intexmw_dfa + inttaxmutf_dfa + mmmf_dfa

gen hwbus_dfa = pthru_dfa + privccorp_dfa 
gen hwequ_dfa = pubccorp_dfa + equmutf_dfa

#delimit ;
gen networth_check_risk = hwfix_dfa + hwbus_dfa + hwequ_dfa + hwhou_dfa +
						  hwpen_dfa + hwoth_dfa + othdebt_dfa;

assert inrange(networth_check_risk / networth_pref, 0.9999, 1.0001) | 
	   abs(networth_check_risk - networth_pref) < 1;
#delimit cr

/********************************************************************************
	(5) Sort and save
********************************************************************************/

drop networth_check_*

isid year category
sort year category

save $dtadir/dfa_revision.dta, replace